#ifndef IO_CC
#define IO_CC
#include<string>
#include<fstream>
#include<sstream>
#include<iostream>
#include<algorithm>
#include<iterator>
#include<vector>
#include<map>
#include<cstdlib>
#include<ctime>
#include<cmath>
#include "def.hpp"
#include "utils.hpp"
#include "io.hpp"

using namespace std;

namespace Puza{
  // The return value of the function is the maximum length of all documents
  int ReadDOCS_Batch(DocList &assignment, int& MAX_INT, int read_mode){
    MAX_INT = 0;
    string temp;
    int TERM_COUNT = 0;
    if(PuzaDEF::Instance()->input_file_name.empty()){
      cout << "Please give input file name." << endl;
      exit(0);
    }
    PuzaDEF::Instance()->InputFile.open(PuzaDEF::Instance()->input_file_name.c_str());
    int counter = 0;
    PuzaDEF::Instance()->data_ids.clear();
    string original_id;
    vector<string> buffer;
    while(!PuzaDEF::Instance()->InputFile.eof()){
      getline(PuzaDEF::Instance()->InputFile,temp,'\n');
      std::istringstream iss(temp);
      // get the doc id
      getline(iss,temp,'\t');
      original_id = temp;
      // get the doc body
      getline(iss,temp,'\t');
      from_string<int>(TERM_COUNT,temp,std::dec);
      getline(iss,temp,'\n');
      std::istringstream tempiss(temp);
      vector<string> tokens;
      copy(istream_iterator<string>(tempiss), istream_iterator<string>(), back_inserter<vector<string> >(tokens));
      if(tokens.size()<1){
	continue;
      }
      
      int now_id = PuzaDEF::Instance()->data_ids.size();
      PuzaDEF::Instance()->data_ids[now_id] = original_id;
      TermList assignList;
      int term_count = 0;
      for (vector<string>::iterator iter = tokens.begin(); iter < tokens.end(); iter++){
	int word_id;
	int word_count;
	std::istringstream valueiss((*iter));
	getline(valueiss,temp,':');
	from_string<int>(word_id,temp,std::dec);
	getline(valueiss,temp);  // get count
	from_string<int>(word_count,temp,std::dec);
	if(read_mode == 0){
	  for(int j=0;j < word_count; j++){
	    TermItem newAssign;
	    newAssign.term_id = word_id;
	    newAssign.term_stat = 0; // initally assigned to topic 0 but later would be changed to a random number
	    assignList.push_back(newAssign);
	    term_count += 1;	  
	  }
	}
	else{
	  TermItem newAssign;
	  newAssign.term_id = word_id;
	  newAssign.term_stat = word_count;
	  assignList.push_back(newAssign);
	  term_count += 1;
	}
      }
      if(term_count > MAX_INT)
	MAX_INT = term_count;
      assignment[counter] = assignList;
      counter ++;
    }
    PuzaDEF::Instance()->InputFile.close();
    return 0;  
  }
  
  int ReadDOCS_Online(DocList &assignment, int& MAX_INT, int read_mode){
    // the key for online reading is that we don't close the input stream unless we detect the end of the input
    string temp;
    string original_id;
    vector<string> buffer;
    if(!PuzaDEF::Instance()->InputFile.is_open()){
      // first time
      PuzaDEF::Instance()->InputFile.open(PuzaDEF::Instance()->input_file_name.c_str());
    }
    int c = 0;
    int TERM_COUNT=0;
    assignment.clear();
    PuzaDEF::Instance()->data_ids.clear();
    while(!PuzaDEF::Instance()->InputFile.eof()){
      getline(PuzaDEF::Instance()->InputFile,temp,'\n');
      std::istringstream iss(temp);
      // get the doc id
      getline(iss,temp,'\t');
      original_id = temp;
      //    cout << original_id << endl;
      // get the doc body
      getline(iss,temp,'\t');
      from_string<int>(TERM_COUNT,temp,std::dec);
      getline(iss,temp,'\n');
      std::istringstream tempiss(temp);
      vector<string> tokens;
      copy(istream_iterator<string>(tempiss), istream_iterator<string>(), back_inserter<vector<string> >(tokens));
      if(tokens.size()<1){
	continue;
      }
      
      int now_id = PuzaDEF::Instance()->data_ids.size();
      PuzaDEF::Instance()->data_ids[now_id] = original_id;
      TermList assignList;
      int term_count = 0;
      for (vector<string>::iterator iter = tokens.begin(); iter < tokens.end(); iter++){
	int word_id;
	int word_count;
	std::istringstream valueiss((*iter));
	getline(valueiss,temp,':');
	from_string<int>(word_id,temp,std::dec);
	getline(valueiss,temp);  // get count
	from_string<int>(word_count,temp,std::dec);
	if(read_mode == 0){
	  for(int j=0;j < word_count; j++){
	    TermItem newAssign;
	  newAssign.term_id = word_id;
	  newAssign.term_stat = 0; // initally assigned to topic 0 but later would be changed to a random number
	  assignList.push_back(newAssign);
	  term_count += 1;	  
	  }
	}
	else{
	  TermItem newAssign;
	  newAssign.term_id = word_id;
	  newAssign.term_stat = word_count;
	  assignList.push_back(newAssign);
	term_count += 1;
	}
      }
      if(term_count > MAX_INT)
	MAX_INT = term_count;
      assignment.push_back(assignList);
      
      c+= 1;
      if(c >= PuzaDEF::Instance()->BatchSize)
	break;
    }
    if(PuzaDEF::Instance()->InputFile.eof()){
      PuzaDEF::Instance()->InputFile.close();
      return 0;
    }
    else{
      return 1;
    }
  }
  
  int ReadDOCS(DocList &assignment, int& MAX_INT,int read_mode, int learning_mode){
    if(learning_mode == 0){
      return ReadDOCS_Batch(assignment,MAX_INT,read_mode);
  }
    else{
      return ReadDOCS_Online(assignment,MAX_INT,read_mode);
    }
  }
};

#endif
